#inputFileName = "/data1/team/weijiang/dataset/gov2-related/06.efficiency_topics.test.wei.head.10"
inputFileName = "/data1/team/weijiang/dataset/gov2-related/06.efficiency_topics.all"


'''
  // All the words in the lexicon are lower case, so queries must be too, convert them to lower case.
  // Wei Added: What about some capital letter can indicate something important? Like a human's name?
  for (size_t i = 0; i < query_line.size(); i++) {
    if (isupper(query_line[i]))
      query_line[i] = tolower(query_line[i]);

    // We need to remove punctuation from the queries, since we only index alphanumeric characters and anything separated by a non-alphanumeric
    // character is considered a token separator by our parser. Not removing punctuation will result in the token not being found in the lexicon.
    int int_val = query_line[i];
    if (!((int_val >= 48 && int_val < 58) || (int_val >= 65 && int_val < 91) || (int_val >= 97 && int_val < 123) || (int_val == 32))) {
      query_line[i] = ' ';  // Replace it with a space.
    }
  }
'''

outputFileName = inputFileName + ".query.terms"

inputFileHandler = file(inputFileName, 'r')
outputFileHandler = file(outputFileName, 'w')

queryTerms = []

for currentLine in inputFileHandler.readlines():
    print "query number:",currentLine.strip().split(":")[0]
    
    # follow the query line normalization rules in the polyIRTK.
    # Rule1: All the terms in the lexicon are lower case, so queries must be too, convert them to lower case.
    queryLine = currentLine.strip().split(":")[1].lower()
    
    print "queryLine1:",queryLine
    
    # We need to remove punctuation from the queries, since we only index alphanumeric characters and anything separated by a non-alphanumeric
    # character is considered a token separator by our parser. Not removing punctuation will result in the token not being found in the lexicon.
    for i in range(0,len(queryLine)):
        int_val = ord( queryLine[i] )
        #if (!((int_val >= 48 && int_val < 58) || (int_val >= 65 && int_val < 91) || (int_val >= 97 && int_val < 123) || (int_val == 32))):
        if not (int_val >= 48 and int_val < 58 or int_val >= 65 and int_val < 91 or int_val >= 97 and int_val < 123 or int_val == 32):
            queryLine = queryLine[0:i] + ' ' + queryLine[i+1:len(queryLine)] #Replace it with a space.
            
    print "queryLine2:",queryLine
    
    for currentQueryTerm in queryLine.split(" "):
        if currentQueryTerm.strip() not in queryTerms and currentQueryTerm.strip() != '':
            queryTerms.append(currentQueryTerm.strip())
        else:
            print "skip ",currentQueryTerm.strip()
      


queryTerms.sort(cmp=None, key=None, reverse=False)
print queryTerms
print len(queryTerms)

for queryTerm in queryTerms:
    outputFileHandler.write(queryTerm + "\n")


inputFileHandler.close()
outputFileHandler.close()